{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Solve MoutainCar-v0 using SARSA($\\lambda $) + TileCoding\n",
    "This notebook solves 'MountainCar-v0' within merely 75 episodes using SARSA($\\lambda $) and tile coding.\n",
    "\n",
    "**Method**\n",
    "- SARSA($\\lambda $) with $\\lambda = 0.9$ and replacement trace. Learning rate $\\alpha = 0.03$.\n",
    "- No discount $\\gamma = 1$. Exploration $\\epsilon = 0.001$.\n",
    "- Tile coding with asymmetrical offsets. $8$ layers and $\\le 2000$ features.\n",
    "- No reward engineering.\n",
    "\n",
    "\n",
    "This solution has been published in the following book:\n",
    "\n",
    "    @book{xiao2019,\n",
    "     title = {Reinforcement Learning: Theory and {Python} Implementation},\n",
    "     author = {Zhiqing Xiao}\n",
    "     year = 2019,\n",
    "     publisher = {China Machine Press},\n",
    "    }\n",
    "\n",
    "\n",
    "**Reference**\n",
    "- Ana Mohamed's solution on MountainCar-v0, which also uses SARSA($\\lambda $) and tile coding: https://github.com/amohamed11/OpenAIGym-Solutions ."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "%matplotlib inline\n",
    "import numpy as np\n",
    "np.random.seed(0)\n",
    "import gym\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Construct Agent"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "class TileCoder:\n",
    "    def __init__(self, layers, features):\n",
    "        \"\"\" \n",
    "        Parameters\n",
    "        - layers: int, the number of layers in tile coding\n",
    "        - features: int, the number of features, also the shape of weights\n",
    "        \"\"\"\n",
    "        self.layers = layers\n",
    "        self.features = features\n",
    "        self.codebook = {}\n",
    "    \n",
    "    def get_feature(self, codeword):\n",
    "        if codeword in self.codebook:\n",
    "            return self.codebook[codeword]\n",
    "        count = len(self.codebook)\n",
    "        if count >= self.features: # collide when codebook is full\n",
    "            return hash(codeword) % self.features\n",
    "        else:\n",
    "            self.codebook[codeword] = count\n",
    "            return count\n",
    "        \n",
    "    def __call__(self, floats=(), ints=()):\n",
    "        \"\"\" \n",
    "        Parameters\n",
    "        - floats: tuple of floats, each of which is within [0., 1.]\n",
    "        - ints: tuple of ints\n",
    "        Returns\n",
    "        - features : list of ints\n",
    "        \"\"\"\n",
    "        dim = len(floats)\n",
    "        scaled_floats = tuple(f * self.layers * self.layers for f in floats)\n",
    "        features = []\n",
    "        for layer in range(self.layers):\n",
    "            codeword = (layer,) + tuple(int((f + (1 + dim * i) * layer) / self.layers) \\\n",
    "                    for i, f in enumerate(scaled_floats)) + ints\n",
    "            feature = self.get_feature(codeword)\n",
    "            features.append(feature)\n",
    "        return features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "class SARSAAgent:\n",
    "    def __init__(self, env, layers=8, features=2000, gamma=1.,\n",
    "                learning_rate=0.03, epsilon=0.001):\n",
    "        self.action_n = env.action_space.n\n",
    "        self.obs_low = env.observation_space.low\n",
    "        self.obs_scale = env.observation_space.high - env.observation_space.low\n",
    "        self.encoder = TileCoder(layers, features)\n",
    "        self.w = np.zeros(features)\n",
    "        self.gamma = gamma\n",
    "        self.learning_rate = learning_rate\n",
    "        self.epsilon = epsilon\n",
    "        \n",
    "    def encode(self, observation, action):\n",
    "        states = tuple((observation - self.obs_low) / self.obs_scale)\n",
    "        actions = (action,)\n",
    "        return self.encoder(states, actions)\n",
    "    \n",
    "    def get_q(self, observation, action):\n",
    "        features = self.encode(observation, action)\n",
    "        return self.w[features].sum()\n",
    "    \n",
    "    def decide(self, observation):\n",
    "        if np.random.rand() < self.epsilon:\n",
    "            return np.random.randint(self.action_n)\n",
    "        else:\n",
    "            qs = [self.get_q(observation, action) for action in range(self.action_n)]\n",
    "            return np.argmax(qs)\n",
    "        \n",
    "    def learn(self, observation, action, reward, observation_next, done, action_next=None):\n",
    "        u = reward\n",
    "        if not done:\n",
    "            u += (self.gamma * self.get_q(observation_next, action_next))\n",
    "        delta = u - self.get_q(observation, action)\n",
    "        features = self.encode(observation, action)\n",
    "        self.w[features] += (self.learning_rate * delta)\n",
    "\n",
    "\n",
    "class SARSALambdaAgent(SARSAAgent):\n",
    "    def __init__(self, env, layers=8, features=2000, gamma=1.,\n",
    "                learning_rate=0.03, epsilon=0.001, lambd=0.9):\n",
    "        super().__init__(env=env, layers=layers, features=features,\n",
    "                gamma=gamma, learning_rate=learning_rate, epsilon=epsilon)\n",
    "        self.lambd = lambd\n",
    "        self.z = np.zeros(features)\n",
    "        \n",
    "    def learn(self, observation, action, reward, observation_next, done, action_next=None):\n",
    "        u = reward\n",
    "        if not done:\n",
    "            u += (self.gamma * self.get_q(observation_next, action_next))\n",
    "            self.z *= (self.gamma * self.lambd)\n",
    "            features = self.encode(observation, action)\n",
    "            self.z[features] = 1. # replacement trace\n",
    "        delta = u - self.get_q(observation, action)\n",
    "        self.w += (self.learning_rate * delta * self.z)\n",
    "        if done:\n",
    "            self.z = np.zeros_like(self.z)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def play_sarsa(env, agent, train=False, render=False):\n",
    "    episode_reward = 0\n",
    "    observation = env.reset()\n",
    "    action = agent.decide(observation)\n",
    "    while True:\n",
    "        if render:\n",
    "            env.render()\n",
    "        observation_next, reward, done, _ = env.step(action)\n",
    "        episode_reward += reward\n",
    "        if done:\n",
    "            if train:\n",
    "                agent.learn(observation, action, reward, observation_next, done)\n",
    "            break\n",
    "        action_next = agent.decide(observation_next)\n",
    "        if train:\n",
    "            agent.learn(observation, action, reward, observation_next, done, action_next)\n",
    "        observation, action = observation_next, action_next\n",
    "    return episode_reward"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "D:\\Programs\\Anaconda3\\lib\\site-packages\\gym\\envs\\registration.py:14: PkgResourcesDeprecationWarning: Parameters to load are deprecated.  Call .resolve and .require separately.\n",
      "  result = entry_point.load(False)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[33mWARN: gym.spaces.Box autodetected dtype as <class 'numpy.float32'>. Please provide explicit dtype.\u001b[0m\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYMAAAD8CAYAAACVZ8iyAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAIABJREFUeJztnX2UZHdZ5z9PvVd1T0/3ZCYJZDJMgtE1QAQyRoKsKKIEj4IE2RPcPeLqOUEPuLrHPUoOx104R11ZPfiyRg85iK6rguhuNhECI+9vIiQhARImgckbGSaZnsl0d/V0vVf99o97f7duVd1bdev9VvfzOadPum/dqnq6a3Kf+3yfNzHGoCiKouxtEvM2QFEURZk/6gwURVEUdQaKoiiKOgNFURQFdQaKoigK6gwURVEU1BkoiqIoqDNQFEVRUGegKIqiAKl5GxCVgwcPmqNHj87bDEVRlIXh3nvvPWeMORTl3IVxBkePHuWee+6ZtxmKoigLg4g8EfVclYkURVEUdQaKoiiKOgNFURQFdQaKoigK6gwURVEU1BkoiqIoqDNQFEVRUGegKIoyMz798DpPni/N24xA1BkoiqLMiP/0/vt4z2cfmbcZgagzUBRFmQHGGHZqTZ7eqs7blEDUGSiKosyAetPQbBnOblfmbUog6gwURVFmQLnWBODstkYGiqIoe5Zy3XUGF6oYY+ZsTS/qDBRFUWaAdQb1pmGjVJ+zNb2oM1AURZkBpVrD+349hnkDdQaKoigzoOJGBgDrxfjlDdQZKIqizIBSre0M4phEVmegKIoyA8o+Z7C+25yBiLxBRB4UkZaIHOt67BYROSkiD4vIq3zHb3CPnRSRt43z/oqiKItC2S8T7cKcwQPAjcBn/QdF5GrgJuB5wA3An4lIUkSSwK3Aq4GrgTe65yqKouxqbGSQTydjGRmkxnmyMeYEgIh0P/Ra4APGmCrwmIicBK5zHztpjHnUfd4H3HO/MY4diqIoccdGBkcOFDi7hxLIlwFP+n4+5R4LOx6IiNwsIveIyD1nz56diqGKoiizwDqD51xU4OyFBXQGIvJxEXkg4Ou1/Z4WcMz0OR6IMeY2Y8wxY8yxQ4cODTJVURQltpRrTRICh9cKrBfjlzMYKBMZY145wuueAi73/XwYOO1+H3ZcURRl11KuNcmnk1y8kmWn1mSn2mApO5ZSP1GmJRPdCdwkIlkRuQK4CvgycDdwlYhcISIZnCTznVOyQVEUJTaU603ymSSHlrNA/MpLxy0tfZ2InAKuBz4sIscBjDEPAh/ESQx/FHiLMaZpjGkAbwWOAyeAD7rnKoqi7GrKNccZXLziOoOYSUXjVhPdDtwe8tjvAL8TcPwu4K5x3ldRFGXRKNddmWhfDiB2SWTtQFYURZkBJZsz2GcjA3UGiqIoew6bM1gtpEknZXflDBRFUZRoVFyZSEQ4tJyN3UgKdQaKoigzoOQmkAEOreRiN7lUnYGiKMoMcPoMnJqdi/dl1RkoiqLsRSr1JvmMc8m9eF9WcwaKoih7kVKtSSHjRAaH9mU5v1Oj1mjN2ao26gwURVGmjDGGcr1JLu3kDGyvwbkY9RqoM1AURZkyVTcCyHvOIH4jKdQZKIqiTBm7/7jgVhPZkRRxSiKrM1AURZkydpdBvksmilOvgToDRVGUKVOuNQDIuZHBRcsZROI1kkKdgaIoypQp15ycQcGNDNLJBAcKGc0ZKIqi7CU8mciNDMApLz2rMpGiKMreoWRlonTbGVwcs5EU6gwURVGmTKXeWU0EuMPq1BkoiqLsGbqricApLz27XaXVMvMyqwN1BoqiKFOmu88AnMazRsuwUaqFPu9fTp7j3R/75tTtA3UGiqIoU6fsOoNchzOwvQbBUtE/ffU0b/rLL/PRB55ip9qYuo3qDBRFWXiMMfz3j5zgkbMX5m1KINYZ+GWiQ/vCu5Df9/nH+JX338eLLl/jH978UpayY62rj8T030FRFGXKnN+p8Z7PPMrBpSzPPbQ8b3N6KNebpJNCOtm+/w6aT9RqGd51/CHe85lHedXzLuGPb3pRRwXSNFFnoCjKwmMTtLVmfEZC+/FPLLXY+UTr2xWMMXzpsfPc+qmTfO5b5/gPLznCO1/zfJIJmZmN6gwURVl4bOlmNUb7Afw4W846nUEhk2I5m+JTD63zka8/zde/s8WBpQy/9ZNX8ws/eBSR2TkCUGegKMouwI57iNOyGD/lerOjkshy6f4cdz++wZUHl/id1z2f17/48MxkoW7UGSiKsvCUvcigOWdLginXemUigHe9/hqK5Tov/+5DJGYoCQWhzkBRlIXHyxnEODLIB0QG1z5nbQ7WBKOlpYqiLDy2dDO2zqAWLBPFibGcgYi8QUQeFJGWiBzzHf8xEblXRL7u/vcVvseudY+fFJE/kVlnSRRF2XVUFqCaqDuBHDfGjQweAG4EPtt1/BzwU8aYFwBvAv6377E/B24GrnK/bhjTBkVR9jixl4lqTfKZeKvyY1lnjDkB9JRAGWPu8/34IJATkSxwAFgxxnzRfd5fAz8NfGQcOxRF2dvEXiaqN8mn463Kz8K61wP3GWOqwGXAKd9jp9xjiqIoIxP3prNSQJ9B3BgYGYjIx4FLAx56uzHmjgHPfR7wLuDH7aGA00Lnt4rIzTiSEkeOHBlkqqIoe5TYN53Vd4FMZIx55SgvLCKHgduBnzPGPOIePgUc9p12GDjd571vA24DOHbsWDyGfiuKEjviLBM1W4ZaoxX7yGAqMpGIrAIfBm4xxnzBHjfGPAVsi8hL3CqinwP6RheKoiiDiHMCub3/eBfnDETkdSJyCrge+LCIHHcfeivwXcBvicj97tfF7mO/DLwXOAk8giaPFUUZkzjnDLzx1YsuE/XDGHM7jhTUffy3gd8Oec49wPPHeV9FURQ/lRhHBpWAlZdxJN5xi6IoSgTinDMoBSy2iSPqDBRFWXhiLRPVe/cfxxF1BoqiLDzlenxHWHv7jzUyUBRFmS6VGMtE5bqzzF4jA0VRlCnjl4mMiVdLkl28EzTCOk6oM1AUZeGxzgDilzco1ZzIQBPIiqIoU8bKRBA/qcgrLdXIQFEUZbqU6032ZZ22qWk5g9ObZX73rhO0WsPJUGXtM1AURZk+9WaLRsuwkk8D05OJPvnQOrd99lFObZSHel5Jq4kURVGmj73z3m+dwZQiA6v9+/MTUSjXm2RTCZJzXng/CHUGiqIsNDZfMH1n4LzPsM6gUmvGPl8A6gwURVlw7MV5teA4g2ntNLDNY+XacM5gERbbgDoDRVEWnB6ZaEo5gx1XJqqMIBNpZKAoijJlyl0yUbUeM5morpGBoijK1LEX52lXE5WqzvuURpCJ4j6KAtQZKIqy4FRmVU1UHy0yKNebsS8rBXUGiqIsOHb2z7SdQdnmDIaMDMqaQFYURZk+vQnk4S7WUdmpjh4ZqEykKIoyZWbVdFYeVSbSPgNFUZTpM6ums52q24E8gkykOQNFUZQp0x0ZxK3pTGUiRVGUGVCuN8kkE54UM43SUmPMSNVEdoieJpAVRVGmjCPDJMgkncvZNGSiaqNF0x1dPYwzWJSJpaDOQFGUBafijntIJIRUQqbiDPzS0DDjKOy5hUxq4jZNGnUGiqIsNGXfuIdMKjEVZ2DnEsFwOQN7bj4T/0tt/C1UFGWmGGN4eqsybzMi46/WyaQSU8kZ+B3AMOMo7Ln5tEYGiqIsGHc/vsH1v/cJTq5vz9uUSPingmaS04kM7EV9KZMcSibyVl5qNZGiKIvG08UKxsCDp4vzNiUSlRnKRBctZ4dKIHsy0W5PIIvIG0TkQRFpicixgMePiMgFEfkvvmM3iMjDInJSRN42zvsrijJ57J3vY+d25mxJNLpzBtUpykQHljLDOQMvgbzLnQHwAHAj8NmQx/8Q+Ij9QUSSwK3Aq4GrgTeKyNVj2qAoygSpLpozqDXJTVkm2nGdwcHlzHAJ5PrilJaOldUwxpwAEOld9CwiPw08Cvj/RV0HnDTGPOqe8wHgtcA3xrFDUZTJUXGXwzy+IM6gUm95kUF2SjKRnVh60VKWaqNFq2VIRFhwb5+3Z3MGIrIE/Cbwzq6HLgOe9P18yj2mKEpMsDLRo+d2MMbM2ZrBzKK01CaQDyxnvPeMZJv7vMJuiAxE5OPApQEPvd0Yc0fI094J/KEx5kJX1BDkSkP/tYnIzcDNAEeOHBlkqqIoE6DScC5g25UGz+zUOLicnbNF/fFPBc2kEl5kM0msM7hoqe0MlrKDhZWya8siRAYDfxtjzCtHeN0fAH5GRP4HsAq0RKQC3Atc7jvvMHC6z3vfBtwGcOzYsfjfoijKLsB/MX3s3E6snYExpmOTWCaZoFhuDHjW8JRqDZIJ8VZrRs0blGsNRBz5Ku5MpRPCGPNv7fci8g7ggjHmT0UkBVwlIlcA3wFuAn52GjYoijIa/jr6x87u8P1HD8zRmv7YCaVTLy2tOpNHbVVQ1F4DK2EF5VXjxrilpa8TkVPA9cCHReR4v/ONMQ3grcBx4ATwQWPMg+PYoCjKZKnUW1y6kiOdFB6NeRK5XcfvXMoyqeTUOpALmaTndCLnDOqLsfISxq8muh24fcA57+j6+S7grnHeV1F2M5U5L1CvNJoUskmWc0s8du7C3OyIQneH79Q6kOtNCpmU9z5RR1KUFmTLGWgHsqLEim8/U+L5/+04D3xna242VGpNcqkkVxxcin2vQXcdfyaVmMpym1K1MVJkUFmgyECdgaLEiCfO79BoGZ48X5qbDZWGsx/gyoNLPP5MyZvjH0e6xz04fQbDbSKLQsnKRDZnoJGBoijTxFbCDLt0fZJU6i1yaScyqDVanN4sz82WQVS6ZaJpRQa1hiMTDZszqGlkoCjKCBQrdWDezqDpOQOI91gKL2fgKy2tNVsTb5YrjZhAtot3FgF1BooSI7bKrjMYcun6JHGcQYIrDi2AM+haK5lNJTAGGhOWthxn0E4gR/18ShoZKIoyCkXXGQwzM3/SVOotcqkkh5azLGdT8XYGATIRTH4PsiMTJT2nE7npTCMDRVFGIQ4yUbXRJOs2Sl1xcCnWvQaVbploas7AkYnSyQTppGg1kaIo02XLJpBrk0+CRsVJIDuXBqe8NL69Bt3VRJ4zmGDjWbNlqDZa3lL7XDoZ2RmoTKQoykhYmSgOCWRwnMGpjTLVKZRrToLuQXCZ5OQjg5I7htqOosino62+tHOTFmGxDagzUJRYYWWieeUMGs0WjZYhl3IuYFceWsIYpxkujlinaQfB2chgkuWlttu4kHWdQSYZKWdQbbQwBm/xTtxRZ6AoMaI452qiinsR9ctEQGzzBpWuQXDZKeQMPGfgiwyijKOwn+VKLj0xW6aJOgNFiRFbc246627iOhrzXoNyV4fvNHIGVibKp52cQT4TLWewUXKcwWpBnYGiKEMy72oi6wysTLSSS3NwOctjZ2PqDLqqdTJJ5/tpRAZL2eFyBhulGgBrhczEbJkm6gwUJSZU6k3vIjavnIFdbJNNty8NV8Z4YF3ZbZCzTKO0NEgmiuKsN11noJGBoihDYTVmmGPOoGsKKNDTa1BvtvjiI8/MtTHOUgmViSZnW6lqq4nc0tKICWQrEy1KZDCVTWeKogyPlYgyycTcZCJbQtrhDA4tce6eKt/ZLPOhr57mr/7lcZ7aqvB7N76Am66b727yXploRpFBJGewWDKROgNFiQk2eXzxSnbuMlHOt7PXVhT9yO9/mlqzxXVXHOCprQrn3YvdPCnXmyz7FtNPpbS0K6leiJhA3izVyaYSOo5CUZThsJHBJSu5WMlEL7x8lctW8/zECy7lQ7/yMv7+5peQTAil6vxlou4R0VMpLXVloiVXJoqaM9jYqS1MVAAaGShKbLA5g0tWstz37SbGmJkvUvciA98F9pKVHF942ys6zitErLWfNt0joqdTWto58iKXTlKpt2i1DIlE+OezUaovTPIYNDJQlNjQdgY5WmayF7SotNdI9r80FLJJr/5+nswmZ9Agn056F37rfAZJUZulxYoM1BkoSkwoVtycwb4cAJU5DKsLkomCKGRSsYgMyrVmh63TKi31zxeKuuBmo1RjbUkjA0VRhmSrXCeXTrA/71xA5lFR1N10FkYhE4/IoFJvBctEE3QG3V3O9vtBv/9mqc6qRgaKogxLsVxnJZcmn3H+t5yHM7DSR3aATLSUSbEz5wRyo9mi1mx1yESphCAyWYltp9bwksfQjgz6VXwZY9gs11nTnIGiKMNSrNTZn0+3ZYg5yDCVehORdlVOGPlM0iu5nBd2qJ7fGYiIswd5wjJRPkgm6iPjFSsNmi2jOQNFUYZnq1xnJZ9ur1ack0yUTSUGVjEtZZNeyeW88PYfd9XxZ1KJiY+wtnOJoC0T9ft82qMo1BkoijIkxXKDlVwqkgwxLZwtZ4ObpOKQQO5eeWnJphITLy21E0uBSM66PYpCZSJFUYakWHEiA+/Oc04y0aDkMcQjgVwOcQaTlonKtUZwNVGf339DIwNFUUalWO7KGcwjMmi0BvYYgBMZ7Mw5MvD2H2c67Z20TLTTJRMVhpCJ9kxkICJvEJEHRaQlIse6HrtGRL7oPv51Ecm5x691fz4pIn8is26xVJQYYoyhWGmwkpt/ziCKTLSUSVJrtGjMoTHOUg7picikEtQmuLO53CUTtSO38N99Y2exJpbC+JHBA8CNwGf9B0UkBfwN8EvGmOcBPwzY+bx/DtwMXOV+3TCmDYqy8OzUmjRbhpV8yrvYzCdn0CQbwRl4tfZzrCgKlYlSk5OJjDFOaakvMojirDdLNURgJb9HIgNjzAljzMMBD/048DVjzFfd854xxjRF5FnAijHmi8YYA/w18NPj2KAouwE7imLepaXVeqtjYmkYS+6k0HkOq2vLRAE5gwlFLHapfVBpaT9nvVFyJL9kn9lFcWNaOYPvBoyIHBeRr4jIb7jHLwNO+c475R5TlD3Nlm95+lxlokY0magQsQt3mpRr048MvF0GvvdIJ8WZ2joggbxIEhFEmFoqIh8HLg146O3GmDv6vO7LgO8HSsAnROReoBhwrunz3jfjSEocOTLfJRqKMk1sZLDi3k1mUvNZcFPpGvwWht36Nc/y0nCZKMnWhHYt7NgtZ76dCSJCIZ3smzPYXLCJpRDBGRhjXjnC654CPmOMOQcgIncBL8bJIxz2nXcYON3nvW8DbgM4duxYqNNQlEXHDqlbyTkXkHw6SWUupaXRqomW3MhgZ46NZ94cpUxvn8Gkqomswyl0vUduwIKbjVKNS1ZyE7FhVkxLJjoOXCMiBTeZ/HLgG8aYp4BtEXmJW0X0c0BYdKEoewZ/zgCiL1CZNFGriWKRQO4nE00oZ7DTtdjGkk8n++YMFjEyGLe09HUicgq4HviwiBwHMMZsAO8G7gbuB75ijPmw+7RfBt4LnAQeAT4yjg2KshvwcgZ5d5tWJkm5Pp8R1pFKS+OQQK43SSWEdLLzMpadYNNZWJJ60B7kXZkz6Icx5nbg9pDH/gZHFuo+fg/w/HHeV1F2G3blpd3nm4u4dH3SVBqtgRNLoS2b7MwzgRyS35hKAnkImajaaFKqNReq4Qy0A1lRYkGx3GA5myLl3uXm04mZ9xm0WoZaoxVxHIXjtOa1qxncKCZg2fxEZSLX2RV6ZKJE6O++6c4lWqRRFKDOQFFiwZY7isKSH5CgnAY26TpMaelcI4NaSGQQIhN9+5kSZ7erQ78H9EYGhUwq9PM5v2NHUagzUBRlSIqVOvtynQtUZn3XHXX/MTgVOwmZf85gGJnozX9zL79714mh3sPOXwpKIIc5g40FnEsE6gwUJRYU3V0GltyAapVpEHX/MTi19ktTHGNdrNT50NdCq84BKNdboTJRo2VotTqr0Z/eKrO+XRnKDjuZtDuB3C+nozKRoigjU6w0OmWiOZSWVoaIDAAK2emNsb7z/tO89e/u4+mt8It3pdYkH2CrtwfZlzdotQxb5TrbleHsLdWapJPivaYlnwnP6XiRwZJGBoqiDIndf2yZR86g4payRkkgw3QX3Gy4uvtGn07iUJnITcL7G8+2qw1apt3PEZVSSF4in06G/u6bpcWbWArqDBQlFjgy0XxzBpVGdJkIprvgxpba9rt4l+vNHvkG2vub/XkDu1+gOHRk0OipJIJ25ObM2+xkY6dGLp2I/HeMC+oMFGXONFuG7WqjIzLIpZNUG60e3XuaWNkjSp8BOEnVnSklkItl56Ld7+JdrgU3yAXJRPZuvViuB17Aw9ipNSlkAyID10EEjb3YKNUXLioAdQaKMne2K52jKKCdsKxMcEnLIKr16KWl4Ng4rXEUUSKDsKF6mYDIwMpNjZYZSn4r15o9ZaWAl6sIit42S7WFSx6DOgNFmTv2LnilK4EMs23q8hLIEXMGS9kkpSkNqrPjObYGyUSBOQPnmN8Z+F9nmCRyqEzUZ/WlM4pisZLHoM5AUeaOvQte6eozgNnuNGjnDCJWE42RQDamt/TTjxcZVIKdgTEmNGcQFBlYmQiGSyKXQiKDfjsnNlUmUhRlFPy7DCy5Oay+rAwpE42TQP6Df36Yf/eeL4Y+7uUMysGvbzeQ9c8ZtP92/qqkMAcTRJgz6Be5bZRqCzexFNQZKMrc2SoH5Ay8i83sJpcO03QGTmSwM2Jk8PDTF/jmme3QxwfJRNbWqKWlnZHBEDJRNVgm8mYzdTlr28+gkYGiKEPjyURBzmAukUFUmShJrdGiMcJQuM1SjWKlEfjcVst4SfWwu3hvy1lEmWirXEfcdcRDRQb1kMggE5xALlbqtAwaGSjKXufURomHng7a7hqOl0D25wzsxWamzmC4BHJhjAU3m2V7se+9S9+pOQ1iEK7vhy22geA+g41SjUvdzWPD5wx6I4OwnMHGgjacgToDRZkov/eRh/iVv7tvqOdsleskpL3LAHwXm1lWEzWaZJIJEgmJdP44C26sbBPUYex3EGF9BjZx3Tcy6OozuPxAoe9rdtNotqg1Wn1zBt05nUUdRQHqDBRloqwXq6wPOSa5WHGG1Im0L8JhF5tpUo24/9jiRQZDJpGNMV5HsF/Lt2yV2ot+wu7i25NBe+/AvZxBvVMmumQlRyaViCwT2YgnWCayv3vn52N/L+0zUJQ9zkapRrFSH0pH755LBP3r2KdF1JWXFiufDFteulNr0nB1oK1yUGTgXKwPr+VDnYHdGXAg4A48KDLYKNVYzadZyaUjJ5BtxBM2jgJ6I7eNHZWJFEXBuegY079ZqptipdExlwjm13Q2jDNYsgtuhmw8s0PonO97/07WARxeK7BdbdAM6EfY6LNApjuBbCt8VgtpVvKp6JGBt+UsPDLozRks5i4DUGegKBPDGOMlEDcC5I8wurecQf+mpmlRGVImyo+YQPY7ys0Ap7lVbkcG0B7X4cf+fbv/btDrDIqVOsY40o0TGUR1BuEyUSbpLPfplvE2S07+pzvSWwTUGSjKhChW2nex/UYv9zwvQCbKphJIwMVmmlQaQ0YGIyaQ/XmCrT4JZC/hGyDrbJRq7M+nvZ3RfmzOwMpE3rKZfJqVfDpyArntDHplIhEJnCxr7YqahI8T6gwUZUL45Y/zO0M4g0qvMwi72EyTcq0ZuawU2lLWsHuQ/Y4yKIIquj0Bl606kUGQrHN+J3z+T3fTmY0+1pbSrORSgZFGEKWQLWeWoEF9izqKAtQZKMrE8F/kNoeIDLa6dhlYZr3trNJoRR5fDe3IYFiHZS/Oy9lUoExUrNRZzqa8xq0gWWezVGdtKfiim0gI6aR4MpH9XPbnM+wbJoFs9x8HjLAGdzVpQGSwiA1noM5AUSaG3xmcD0iMBlFtNKnUW4Had27GzqA6dDXRaJHBphs1PeeiQqDT3HJlMxstBSXjz+/UONDnDjyTTHjOYMvbSTxsAtmVidK9jhqc3z+o6UwjA0XZ4/gdQNScgR2nvBLgDPKZ5IwH1Q3nDLIpJ4k6dM6gXGcpk+TQvmxgn0Gx7OyD3m8jg4CL96CdAZlUwhtUt+nrSVjJpak1WpH+rmVbTRQSGQRFbou6ywDUGSjKxLAXneVsqiN/0A9vYmlA9cmscwaVeotcKvolQURYGmGM9Wapzmohw1ohw2ZIn8FKPuWN5wiSdc6XaoE9BpZMKuGTidojwq3TjRId7PSpJgI3cguQiRaxrBTUGSjKxDi/UyOZEC4/UIgcGVgJJB45g+EiA3DumoftQN50dfX9+TSbIX0GK7k0S5kUCemVico1R1oLyxkAZFPJtkxUrrOSS5FKJjwHE2XBzYVKg4SEz2rqjtwq9cF2xRl1BooyIZy7wgwXLWUiVxPZMsfAnEEmSbk+2xHWw/QZwGgLbjbdBrDVQprtaoN6V7d2seyM50gkxC0F7XQG/UZRWByZqJ1AttKNjcCi9Bqc3a5y0XI2tEw0n052/O4b3iiKPRgZiMgbRORBEWmJyDHf8bSI/C8R+bqInBCRW3yP3SAiD4vISRF52zjvryhxYmOnzpp7kQvSwoPY9FW6dJNPJ3qqVaaFMcZtOhsyMhhhwY3V1e3FvPvOv1hpeM4xqEnsfJ/uY4s/gezIUu7ruRFYlF6D9e0KF+/Lhj7eHbnd+8QGAEfc/ohFY9zI4AHgRuCzXcffAGSNMS8ArgXeLCJHRSQJ3Aq8GrgaeKOIXD2mDYoSC86XaqwtZTiwlOF8RJnoTLECwCUrvRedWcpEtiZ/FGewM0LT2Wo+7V2g/Y6z0Wxxodrw7uCd6p/OC7e9Az/QR47JpBIdfQYjRQYXqv2dQZdM9N7PPcbRiwq89LkHB752HBnLGRhjThhjHg56CFgSkRSQB2pAEbgOOGmMedQYUwM+ALx2HBsUJS5suOWOa4UMW+V64Eydbs4UqxQyyY7x1ZZ8QOnitLATPrNDJJDBlYmGsNEY45OJbGTQdpzt6irn77E/n+6JHNo7A6IlkDfdIXXO60ZPIK8Xq1y8Lxf6uD/Bf+8TG9z/5Ca/8LIrSC5g9zFML2fwj8AO8BTwbeAPjDHngcuAJ33nnXKPKcrCs1Gqs7aUZq2Qjjys7kzRkSL846stQU1N06LSGG7lpWUpm6Q0xKA6O3hurZDxLtD+YXX2It1PJvKG1PVNICc6xlFYx9GODPo4zsAzAAATIUlEQVTb3GwZzl2ocmhAZFCuNzHG8Beff5T9+TQ/c+3hvq8bZ4K7KXyIyMeBSwMeersx5o6Qp10HNIFnA2vA59zXCXKZobdPInIzcDPAkSNHBpmqKHPDGVLnJJDtRer8Tq2vlAHu3edK8N3nLGWiYfcfW/Lp4RLIW74Bc55M5LvYt7e++ZxBJThnsBqQdLfYnEGzZShW6ux3o5BcOkE6KQNHUjyzU6Vl4OIA+c6SSydpGXjk7AU++sDTvPnlzw2cY7QoDLTcGPPKEV73Z4GPGmPqwLqIfAE4hhMVXO477zBwus973wbcBnDs2LHBMbeizAk7pO6AmzOAaI1nZ7YrXHN4NfCxfDpJo2WoN1ukAwayTZJh9x9bloYsLfWGxhUynkzk70Jul9r6cgZdd/GbpZpXKhqGlYmKZXdiqft6IuKMpBjgDNaLzoKiQQlkgD/79CMkRHjT9Uf7vmbcmda/sG8DrxCHJeAlwEPA3cBVInKFiGSAm4A7p2SDoswM/4YrW+UyqPHMGMN6scolIRecWS64GXb/saWQSXnNWVHwz/vfl3X6CPwJZHuR9ucMyvVmxz7j86X6wIjLlpb6h9RZVnK9DqabsxccZ3CoX87A/XzuuP80P3nNs7h0f/i5i8C4paWvE5FTwPXAh0XkuPvQrcAyTrXR3cBfGmO+ZoxpAG8FjgMngA8aYx4cxwZFiQP+zVtrESOD7WqDcr3JJSEykZVsZpE3GFUmKmSc5q6om93sxXm14PQRrHZ1Idv8wP4+Cd+NndrAxi4rE3lO2le6G9S70M3ZCJGB7Uxutgy/+LIr+77eIjCWwGWMuR24PeD4BZzy0qDn3AXcNc77Kkrc2PQqXDJesnLQgpt1t6w0TJfOz3DBTcW9885nhq0mai+4WYkgZXXvCF7Npzv+Tl5kkOtO+NY5uOz8nTZKtVAHarEy0aZvSJ0lyoKb9W3ns+mXQLaO8weuOMALDu/v+3qLgHYgK8oE8DdC5dNJsqnEQJnojHv3GXZhm4dMlB1SJhp2wc1m14ay/YW0l1QGJ2eQTIjnZOx5/sqsjZ3awMmgnjMo9y6oX8mnBo6jWN+uspJL9Y2UrHO6+YcWPyqAMSMDRVEcPC18KYOIOI1nA52BbTgLryaC2exBHkcmAiInkTdLzq4CmxBfK2S8u3BwqolWcimv1DaoY3jQkDpwm86aLd+C+vb5+7IRZKLt8Covy4uPrPLJX385Vx5a7nveoqCRgaJMgI2SM6TODkJbLWQGykRnBujSs9yDXB2xmsiWUkYtL7VD6iyr+XRPn4F/TlN3x7AdUjdoTHTWlzMQgX2+qbBBFUrdrG/37z4GpzJptzgCUGegKBPhvDuXyN7RHlhKD0wgnylW2JdNeVJLN1YmmsVOg/KYkcFOxMYz231sWXW7tS1b7pA6S3cCOcooCnAiA4CzF2qs5NIdXcErud4KpW7Wtyt98wW7EXUGijIBunXstUJmYM5gfbvSt6mpLRNNf3Lp2DJRRIe1Wer8O60W0lyoNrwLsx1fbenOGbRzM4NlInCS9N3nWgcT1nhmS34HRQa7DXUGijIBNkqd5Y5rhUyEyKDatypmptVEViYacjbRKAlkvwxkowR7sfdPLAVnrEQmmfBkHX/VVj8ybk7izHbF6z62DJpcWqw0qDZafecS7UbUGSjKBOjecLW2lGFzwLC6M8VKX2eQc8s8Z1Na2iSVkL5dvUFYhxV1D3KQTATtYXXOLoO2bCYiHXuLz0eWiRy71ovV3sgg1z8yOLvt5nL6RG27EXUGijIBzu90dsUeGDCszhjjJCkjyESzajobViKCdmQQpeKp1TK9MlG+sydjq0smAkfWsX/H7j6FMKxMdO5CtWeG0b4Bw+q8HoNldQaKogyBMb0XuUFdyFvlOrUBUsQsq4mcxTbDXw68BHKEyGC72qBlOre6rXnziepU6k2qjVZHAhk6m8S8IXURcwYt0+s42jKRRgZ+1BkoyphsVxs03LHMlkHzidoNZ+EXnHTSmbA5m9LS5tANZ+Bo+gmJljPY8g2ps6x63do13y6D3sjA6vsbO86QukGD+zK+x7sdx6AFN3ZIXb+5RLsRdQaKMiZB8/UP+MZYBzGo4cyS8y1QmSaVxvD7j8HR9Jci7kH2D6mz7LcJ5FK9PbE011lquz+fZrtsS0vrkRbO+5f0dMtEgxbcrG9XyKYSPXbsdtQZKMqYWL3b3xUbtNLRj+cMBtx95tPJmfQZjLL/2JKPuAfZP6TOsi+bIpkQNss138TS7jv5lOcoNkqDR1FAWyaC3iU4S5kkCSF0JMVZN5cTtHBoN6POQFHGZGOnN6npRQYhOYP1iLr0rFZfjppABieJHCUyCEr+iog3rK57YqnFThk1xkRaGASdzqD79ZwKpfBhdevb1T2XPAZ1BooyNt74at9FLp9OkukzrO5MscL+fHrgBTg/K5moPppMBE4SOVJkYHMGXRfnVXdYnc0L9FQT5dLUm4ZK3ZlCOih5DN05g17nsS+XCu0zcEZR7K18AagzUJSx8Q+ps4gIB/o0njk9BoPvPnMzWn1ZqbeGXmxjKWSS7ERIIHdPLLWsun+n9paz3pwBOBVY53dqHU43jA6ZKMB59BtjvV7s3xm+W1FnoChj0j2kzrK2lOH8TljOoH/3sWVmOYPG6DJRIZOKNI5io1RjX8C6ytV8mk2fTNTbZ+D8Xde3K5TrzUgJ5ExHArn3/KDdyuBESMVKY8+NogB1BooyNhulziF1lrVCumO/r5/1YiWSFDGrnEG13iI7oky0lE1SijCobqscLPHYYXXFSp1sKtHjlKxzeOKZEjB4FAW0ZaKEOJJQN2GTS22PwV4bUgfqDBRlbMKWrawtZQITyK2W030cRSaabc5gxGqidPQEctBd+mrBmfBa7JpYarEy0RPP7AAM3GUA7dLS/XlnvWY3K7l04DgKL7GvOQNFUYblfIgzOBAyuXSjVKPRMpFkolw66Q2RmyaVenPknMFSNloCeSMk+buaT1OqNTm7XQus7bcO4vFhIgPXGYSNrdiXSwcmkM9GWHe5W1FnoChjslmqsxZwt7pWcGbqdA+ri9J9bMlnEjPbgTx6NVGKnQiRgSMTBUQGbg7g1EYpMDKwDsJGBtGazhzHFlZ5tJJPcaHaoNHsdLRRS353I+oMFGVMnDWMwTJRy/SOPTjj3n0OWqsIs5GJ6s0WzZbxBuMNSyGTpNZo9VxYu+me7GpZzbdzAt2VRjBmZBDwetDOQ1zoynWc3a6SELhoSZ2BoihDYIxhY6cWeMcb1ni27nYfR6lYybulpcaEj8Iel1EX21iiLLhptYwTGQRcnO3FvVxv9lQSgTOjqZBJesndKH0GyYSQTEio4/BGUnQlkdeLVS5aznZsRtsrqDNQlDG44A6pC6p9X/UmcnY6gzPF6BUrOfdCW+2zonFc2isvR5eJoP+wuu1KA2PoWTQDnRf37h4D77jrJPZFGFJnWc6mOBjyN7bSU3d56fp2ZU+WlQLsrUlMijJh7DL3IB3bOojuXoMzxQoHljKRpoS2V1+OXu0ziKqboM6OPI7CjQz6JJGDhtRZ/NJQkEwEjpN4ujh4qY2f9/38MY4cWAp5veBhdU738d50BhoZKMoYnO9zkbNJ5e6KojND7NedxerL8WUiNzLok9sIGlJn8TvSIJkI2k5i0FIbP9c+50Bo9GV7D3pkoj06igLUGSjKWASNorB4Ow26cwbb/ddd+slnZuEMRtt/bPEW3PRpPLNS2f6APoOlTJKUq9EHVRNB20kciJAviIK308AXGTRbhmcuVPdkWSmoM1CUsdgIGFJnKWScYXXdCeSoc4nAt+1sihVFlcb0E8jtRfa9F3MR8e74wyID6ySilJVGoZ1AbjuDZ3aqtMzeLCsFdQaKMhZ2YmlQ1YqIsFZId8hEzZbh7Ha0uUTg24McY5nI7kHul0AetLvYykdhOQN7PEpZaRT2ZVOI0NF4Zjecac5gBETk90XkIRH5mojcLiKrvsduEZGTIvKwiLzKd/wG99hJEXnbOO+vKPNms1QnmZDA+TfgXLw2SkF3nzGUiUasJrIOq98e5I2QiaUWW3IaXk3kHB8mgdyPREJYzqY6RlK05xJpzmAUPgY83xhzDfBN4BYAEbkauAl4HnAD8GcikhSRJHAr8GrgauCN7rmKspCcdxupgubfgHPx8kcG9u7zkmETyNOUiSYUGfSzcatcZyWXCq3fjywTTSgysO/lTyCvb0fv/9iNjOUMjDH/bIyxf81/BQ67378W+IAxpmqMeQw4CVznfp00xjxqjKkBH3DPVZSFJKzhzLJW6BxWF3X3sSU3y2qiMfYZQP/IYLPU/+80SCayTiIo5zAqzoKboMhgbzqDSfYZ/ALw9+73l+E4B8sp9xjAk13Hf2CCNvTwU//z8zOZB6/sTb6zWeZ5z14JfXxtKc0Tz5T4sXd/BmhXr0RNUlqZ6HfvOsGffvLkmNYGY5fKjCoTZVMJEgLv/dxj3P6V7wSec3qzzHMvXg59DXuRD5PbVkYoLR3ESj7N5791zvtszl6ospJLTa2fI+4MdAYi8nHg0oCH3m6MucM95+1AA/hb+7SA8w3BkUhon72I3AzcDHDkyJFBpgby3ENL1AbMTFGUUbnqkmV+6ppnhz7++hcfZqNU7xgn8ez9eS6NGBk8ayXHz7/0qCdhTItLV/Ij3xGLCP/5ld/NiaeLoedcdckyr3pe0GXE4cYXH+bifbmexTeWl37XRdz8Q1fyoiOrgY+Pwn986VH+6WunO2y89jkHJvb6i4aMO/NERN4E/BLwo8aYknvsFgBjzH93fz4OvMN9yjuMMa8KOq8fx44dM/fcc89YtiqKouwlROReY8yxKOeOW010A/CbwGusI3C5E7hJRLIicgVwFfBl4G7gKhG5QkQyOEnmO8exQVEURRmfcXMGfwpkgY+5K//+1RjzS8aYB0Xkg8A3cOSjtxhjmgAi8lbgOJAE3meMeXBMGxRFUZQxGVsmmhUqEymKogzHzGQiRVEUZXegzkBRFEVRZ6AoiqKoM1AURVFQZ6AoiqKwQNVEInIWeGLEpx8Ezk3QnGmgNk4GtXEyLIKNsBh2ztPG5xhjDkU5cWGcwTiIyD1Ry6vmhdo4GdTGybAINsJi2LkINoLKRIqiKArqDBRFURT2jjO4bd4GREBtnAxq42RYBBthMexcBBv3Rs5AURRF6c9eiQwURVGUPuxqZyAiN4jIwyJyUkTeNm97LCLyPhFZF5EHfMcOiMjHRORb7n/X5mjf5SLyKRE5ISIPisivxs1G156ciHxZRL7q2vlO9/gVIvIl186/d8elzxV3B/h9IvKhONooIo+LyNdF5H4Rucc9FrfPe1VE/lFEHnL/bV4fJxtF5Hvcv5/9KorIr8XJxn7sWmcgIkngVuDVwNXAG0Xk6vla5fFXwA1dx94GfMIYcxXwCffnedEAft0Y873AS4C3uH+7ONkIUAVeYYz5PuCFwA0i8hLgXcAfunZuAL84Rxstvwqc8P0cRxt/xBjzQl8ZZNw+7z8GPmqM+TfA9+H8PWNjozHmYffv90LgWqAE3B4nG/tijNmVX8D1wHHfz7cAt8zbLp89R4EHfD8/DDzL/f5ZwMPzttFn2x3Aj8XcxgLwFZyd2ueAVNC/gznZdhjnIvAK4EM4a2HjZuPjwMGuY7H5vIEV4DHcPGccbeyy68eBL8TZxu6vXRsZAJcBT/p+PuUeiyuXGGOeAnD/e/Gc7QFARI4CLwK+RAxtdOWX+4F14GPAI8CmMabhnhKHz/2PgN8A7DLui4ifjQb4ZxG51909DvH6vK8EzgJ/6cpt7xWRpZjZ6Ocm4P3u93G1sYPd7Awk4JiWTg2BiCwD/wf4NWNM+LbzOWKMaRonLD8MXAd8b9Bps7WqjYj8JLBujLnXfzjg1Hn/2/xBY8yLcWTVt4jID83Znm5SwIuBPzfGvAjYIaZyi5v/eQ3wD/O2ZRh2szM4BVzu+/kwcHpOtkThjIg8C8D97/o8jRGRNI4j+FtjzP91D8fKRj/GmE3g0zg5jlURsStd5/25/yDwGhF5HPgAjlT0R8TLRowxp93/ruPo3NcRr8/7FHDKGPMl9+d/xHEOcbLR8mrgK8aYM+7PcbSxh93sDO4GrnKrNjI4Ydudc7apH3cCb3K/fxOOTj8XxFlo/RfACWPMu30PxcZGABE5JCKr7vd54JU4ScVPAT/jnjZXO40xtxhjDhtjjuL8G/ykMebfEyMbRWRJRPbZ73H07geI0edtjHkaeFJEvsc99KM4O9ZjY6OPN9KWiCCeNvYy76TFlJM4PwF8E0dHfvu87fHZ9X7gKaCOc8fzizg68ieAb7n/PTBH+16GI1t8Dbjf/fqJONno2nkNcJ9r5wPAf3WPXwl8GTiJE6pn5/2Zu3b9MPChuNno2vJV9+tB+/9KDD/vFwL3uJ/3/wPWYmhjAXgG2O87Fisbw760A1lRFEXZ1TKRoiiKEhF1BoqiKIo6A0VRFEWdgaIoioI6A0VRFAV1BoqiKArqDBRFURTUGSiKoijA/wdpLaVPH+OxGwAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "env = gym.make('MountainCar-v0')\n",
    "env.seed(0)\n",
    "env = gym.wrappers.Monitor(env, \"./records\", video_callable=lambda _:True)\n",
    "\n",
    "agent = SARSALambdaAgent(env)  \n",
    "\n",
    "episodes = 75\n",
    "episode_rewards = []\n",
    "for episode in range(episodes):\n",
    "    episode_reward = play_sarsa(env, agent, train=True)\n",
    "    episode_rewards.append(episode_reward)\n",
    "    \n",
    "plt.plot(episode_rewards);"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "average award = -10357.0 / 100 = -103.57\n"
     ]
    }
   ],
   "source": [
    "agent.epsilon = 0.\n",
    "episodes = 100\n",
    "episode_rewards = [play_sarsa(env, agent, train=False) for _ in range(episodes)]\n",
    "print('average award = {} / {} = {}'.format(\n",
    "        sum(episode_rewards), len(episode_rewards), np.mean(episode_rewards)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Plot Policy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "D:\\Programs\\Anaconda3\\lib\\site-packages\\matplotlib\\figure.py:445: UserWarning: Matplotlib is currently using module://ipykernel.pylab.backend_inline, which is a non-GUI backend, so cannot show the figure.\n",
      "  % get_backend())\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXYAAAEKCAYAAAAGvn7fAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAHyNJREFUeJzt3X20HHWd5/H3xyDJgAoJAQx5MGjCuCiCszHMrLMjykOC50gYDRBmRuOIZnCXPTBOdg2LKwqMCx5XmV04qxmMZsTlKQ6Hq0ZzQiDOrkchAeUhYMgl6OaajDwkiyJDMPjdP7qClb51b1d3V3d1V39e59xzu6p+XfXtup1vfv3tX/1KEYGZmVXHK8oOwMzMiuXEbmZWMU7sZmYV48RuZlYxTuxmZhXjxG5mVjFO7GZmFePEbmZWMU7sZmYVc1DZAXTTwZoYkzi07DBKt3dm/56DiTt+XXYINiB+xZ6nI+LIVp+/4J2HxjO7X8rV9r4H966LiIWtHqveQCX2SRzKyTq17DBK9/jyPyw7hJa94ZIflh2CDYg7Y83P2nn+M7tf4t51s3K1nTBt29R2jlXPpRgzs4oZqB57VT1+bf/2wIvWK+fCnyysTO6xm5lVjBO7mVnFOLGbmVWMa+w9pldqxM1yTflAzf4dff6sSO6xm5lVjBO7mVnFuBTTY1r5SN6v5ZtO6ERJo5Xz69KKlck9djOziik1sUtaKGmrpGFJKzK2T5R0S7L9HkmzU9veIukHkrZIekjSpG7GbmbWq0pL7JImANcDZwLHA+dLOr6u2QXAnoiYA3wBuCZ57kHAjcCFEfEm4BTgN10K3cysp5VZY58PDEfEdgBJNwOLgEdSbRYBn0oerwGukyTgDODBiHgAICKe6VbQvahRPbeIGnGjfWRt74U6c31cnYipF16nWVqZpZjpwI7U8kiyLrNNROwDngWOAI4DQtI6SfdL+k9diNfMrC+U2WNXxrrI2eYg4I+BtwHPAxsk3RcRG0YdRFoGLAOYxCFtBWxm1g/KTOwjwMzU8gxg5xhtRpK6+mHA7mT99yLiaQBJa4E/AEYl9ohYCawEeI2m1P/HMRCaLat0yrqdD4y7fcExJzbVPt9zDlxecMmJmFVdmaWYTcBcScdKOhhYAgzVtRkCliaPFwN3RUQA64C3SDokSfjv4MDavJnZwCqtxx4R+yRdRC1JTwBWRcQWSVcAmyNiCPgy8DVJw9R66kuS5+6R9Hlq/zkEsDYivl3KCzEz6zGlXnkaEWuBtXXrPpl6/AJwzhjPvZHakEcrwfC5X2rc6Nzm9pmn9NLuc7La15dzzPqdrzw1M6sYJ3Yzs4pxYjczqxjP7jiOqtwsIc/rGFUzH1Ufb77+3S8a1+kbbM/xXYLr+O1pdqjsoHOP3cysYpzYzcwqZqBKMXtnHsrjy6t/UwrfeKP3NDssc86tfzVqXTduItKNcmKj92dWDI1KLR15z1+8pvh9dol77GZmFePEbmZWMU7sZmYVM1A19qI1HiLYePbBRrXDVi6Bz3W5v1mGRjX3PDX5duvdWc9v9J6ec6u/V0pzj93MrGKc2M3MepykmZLulvSopC2SLh6vvUsxKfUf97KGnDWr0TC3bsxoaNXQqIzXSgmk0Xu+/pj1JQ8Pre2afcDfRMT9kl4N3CdpfURk3ofCPXYzsx4XEbsi4v7k8a+ARxl9j+iXObGbmfURSbOBtwL3jNXGpRgzs/JNlbQ5tbwyuV/zASS9CvgGcElE/HKsnQ1UYj9h8lPc66GA1gfyDFlttr6dtc9G3yMV8T1TEXoljg56OiLmjddA0iupJfWvR8Q/jtfWpRgzsx4nSdTuAf1oRHy+UXsndjOz3vd24P3AuyT9OPl591iNB6oUY9YvWik9VOWK4yJeR9VKNxHxfwDlbe8eu5lZxZSa2CUtlLRV0rCkFRnbJ0q6Jdl+TzLMJ719lqTnJC3vVsxmZr2utFKMpAnA9cDpwAiwSdJQ3ZVUFwB7ImKOpCXANcB5qe1fAL7TyvGr9lHNqqVbZZXGk2t1/t9Jntfa9D1Nr20xmIoos8c+HxiOiO0R8SJwM7Cors0iYHXyeA1wavLtMJLOBrYDW7oUr5lZXygzsU8HdqSWRxh9iezLbSJiH/AscISkQ4GPA5/uQpxmZn2lzMSe9Q1v5GzzaeALEfFcw4NIyyRtlrT5qWdeaiFMM7P+UuZwxxFgZmp5BrBzjDYjkg4CDgN2AycDiyV9Fjgc+K2kFyLiuvqDJJflrgSYOGtmuLZull8nZjxtRbM3ABl0ZSb2TcBcSccCPweWAH9W12YIWAr8AFgM3BURAfzb/Q0kfQp4Liupm5kNotISe0Tsk3QRsA6YAKyKiC2SrgA2R8QQtUtovyZpmFpPfUlZ8ZqZ9YtSrzyNiLXA2rp1n0w9fgE4p8E+PtWR4KjOlXxmZSqibFL/b3HBJU0OfxwwvvLUzKxinNjNzCrGid3MrGI8u+M46od2ueZug66Iy//ra+6c2/i4ze6z0fDIPJ5o+hm9wz12M7OKcWI3M6sYl2Ka4NKM2YHyzLrYeHjjAwXsw9LcYzczqxgndjOzinEpZhwutZiNb93O0WWU+vJMEf+OemUysn7hHruZWcU4sZuZVYwTu5lZxQxUjX3ijl+PewVaVr3QrJFWhvw1uhIya3hf/XN69f3ai3HlGS7ZytWpvco9djOzinFiNzOrmIEqxeydeSiPLx/vI1nvfYS03pev9HBgm0Y3isgcIphjsiwzcI/dzKxynNjNzCrGid3MrGIGqsZuVpZRl8Bf26h94+F5nvKiPVUa3ljPid3MrAMe2nNkE3PaLC/02KWWYiQtlLRV0rCkFRnbJ0q6Jdl+j6TZyfrTJd0n6aHk97u6HbuZWa8qrccuaQJwPXA6MAJskjQUEY+kml0A7ImIOZKWANcA5wFPA++JiJ2S3gysA6Y3c3x/jLW86ntdrXyEH9554Pstz9WqDXn4o42hzB77fGA4IrZHxIvAzcCiujaLgNXJ4zXAqZIUET+KiJ3J+i3AJEkTuxK1mVmPKzOxTwd2pJZHGN3rfrlNROwDngWOqGvzPuBHEbG3Q3GamfWVMr88Vca6aKaNpDdRK8+cMeZBpGXAMoBZ0w9yCcYaavSFV/2VpoWUVVrQKE6/18fX7MRs/aTMHvsIMDO1PAPYOVYbSQcBhwG7k+UZwO3AByLi8bEOEhErI2JeRMw78ogJBYZvZtabykzsm4C5ko6VdDCwBBiqazMELE0eLwbuioiQdDjwbeDSiPh+1yI2M+sDpSX2pGZ+EbURLY8Ct0bEFklXSDorafZl4AhJw8DHgP1DIi8C5gD/RdKPk5+juvwSzMx6UqkXKEXEWmBt3bpPph6/AJyT8byrgKs6HqBZhlZq6s0+J8+NNhrJc3FMv9bhO3Fz61Hn/OI1be+zLJ4rxsysYpzYzcwqxnPFmNWp/5hfX0bJM9yx2ft+1u8jz402Fhwz/jHzlCc8ZLKa3GM3M6sYJ3Yzs4pxYjczqxjX2G3gNTsUMU/7dqcZaOX5RQz5a/YYrsH3JvfYzcwqxondzKxiGpZiJH0O+EpEbOlCPGbWok5cjdkvBvm1Z8nTY/8JsDK5Nd2Fkg7rdFBmZta6hok9Im6IiLcDHwBmAw9K+l+S3tnp4MzMrHm5auzJ/UnfmPw8DTwAfEzSzR2MzczMWpCnxv554D3AXcBnIuLeZNM1krZ2Mjgza12e2SCbnfqgV4c3DnpNvV6ecewPA5+IiOczts0vOB4zM2tTnlLMn9cndUkbACLi2Y5EZWZmLRuzxy5pEnAIMFXSZH53Y+nXAMeM9TyzMpV1Y+leUMQNQJqdITKrNNPu1alZx2y0j0bbB61UM14p5q+AS6gl8ftT638JXN/JoMzMrHVjJvaI+Dvg7yT9h4j4H12MyczM2jBeKeZdEXEX8HNJ763fHhH/2NHIbODlKS00O6qjLHluztGLunFFZys3BPGVpuMbrxTzDmpDHN+TsS0AJ3Yzsx40Xinm8uT3X3YvHDMza1fD4Y6SPiPp8NTyZElXFXFwSQslbZU0LGlFxvaJkm5Jtt8jaXZq26XJ+q2SFhQRj5lZL5K0StKTkh7O0z7PBUpnRsR/3r8QEXskvRv4RKtBwsvTFFwPnA6MAJskDUXEI6lmFwB7ImKOpCXANcB5ko4HlgBvojZq505Jx0XES+3EZP2nX2rVVVF/Nescmq9tF1EP70RNvb5uP+Hiwg/Rjq8C1wH/kKdxnguUJkiauH9B0u8BE8dpn9d8YDgitkfEi8DNwKK6NouA1cnjNcCpkpSsvzki9kbEE8AwvgrWzCoqIv4J2J23fZ4e+43ABklfofal6Yf4XbJtx3RgR2p5BDh5rDYRsU/Ss8ARyfof1j13egExmZn1vYaJPSI+K+lB4LRk1ZURsa6AYytjXeRsk+e5tR1Iy4BlALOm994tXosYxtWrEzM1MkhllCJea6Mhk60MqeyX4aL1iriatcdMlbQ5tbwyIla2urO8me5HwCupJc8ftXqwOiPAzNTyDGDnGG1GJB0EHEbt40ie5wKQnJyVAPNOnJSZ/M3MSvZ0RMwramd5RsWcC9wLLAbOBe6RtLiAY28C5ko6VtLB1L4MHaprMwQsTR4vBu6KiEjWL0lGzRwLzE1iNDMbeHl67JcBb4uIJwEkHQncSe3LzJYlNfOLgHXABGBVRGyRdAWwOSKGgC8DX5M0TK2nviR57hZJtwKPAPuAf+8RMWZWVZJuAk6hVrIZAS6PiC+P2b7WAR53hw9FxAmp5VcAD6TX9Yt5J06Ke9fNKjWGbtT6Gt1goYi6aqP6blYb66x+qZf3ar171HDHadvua6c8MnHWzJi+/JJcbZ+4eHlbx6qXp8f+XUnrgJuS5fOAtUUFYGZmxcozKuY/Snof8HZqo1FWRsTtHY/MzMxakmtUTER8A/hGh2MZCPUf9+rLFY9f+4cHLOe5b2Wzskok7c4+OEhll36dqdEGx3jT9v6K7LHhAiIiXtOxqMzMrGXjze746m4GYmZmxchVipH0x8DciPiKpKnAq5M5WqxgnSi95OFyQn6dOFf9MqIlj26MeunXq627Jc8FSpcDHwcuTVYdTG3+GDMz60F5Znf8U+As4NcAEbETcJnGzKxH5UnsLyaX8QeApEM7G5KZmbUjT439VklfAg6X9BFq0/b+fWfDMhssrczEmOfq3zI0qn+3UoN3Tb05eRL7b4H/DfwSOA74ZESs72hUZmbWsjyJ/dXUblG3m9pdjh7saERmZtaWPFMKfBr4tKS3UJsn5nuSRiLitAZPrbwiPj63294GV6+UXrqh0Q1pGrXP0nho8baG++hVeb483e9J4J+BZ4CjOhOOmZm1K8849o9K2ghsAKYCH4mIt3Q6MDMza02eGvvrgEsi4sedDsbMzNqXp8a+ohuBVJVnSbRW1M/yCTDn1tHr0nplSGCzwxnz3MC92dfWaBbVqmumxm5mZn3Aid3MrGJyze5YFQ/tOXLcj4mD/vHNekfWULys8kxas0MC8+yjkSLKP3mOWcRrGyTusZuZVYwTu5lZxZRSipE0BbgFmA38FDg3IvZktFsKfCJZvCoiVks6BLgNeAPwEvDNvCN3Tpj8FPcmH+FamYjIV4VaP8lTvmj3phjduKmGNa+sHvsKYENEzKV24dOoxJwk/8uBk4H5wOWSJiebPxcRbwTeCrxd0pndCdvMrPeVldgXAauTx6uBszPaLADWR8TupDe/HlgYEc9HxN0AEfEicD8wowsxm5n1hbIS+9ERsQsg+Z0198x0YEdqeSRZ9zJJhwPvodbrNzMzOlhjl3Qn8NqMTZfl3UXGukjt/yDgJuC/R8T2ceJYBiwDmDB5cls1wUY3Nsiqubc7u2MrcVk11Q+BbDT8scoafX8w6P8mOpbYx5vWV9IvJE2LiF2SplGbObLeCHBKankGsDG1vBLYFhHXNohjZdKWibNmxnhtzcyqoKxSzBCwNHm8FLgjo8064AxJk5MvTc9I1iHpKuAw4JIuxGpm1lfKuvL0amr3Ur0A+L/AOQCS5gEXRsSHI2K3pCuBTclzrkjWzaBWzvkJcL8kgOsi4oaig2y2LJKnfaM2g/4R0sbWbumlE0MTOzGE0tpXSmKPiGeAUzPWbwY+nFpeBayqazNCdv3dzMzwladmZpUzUJOAmZl1y8Qdv85xX9WaJwo+thN7SrszyBVRW3wD+d4INnj6Zbhjoxtn5LmxhrXHpRgzs4pxYjczqxiXYsysZa3cn7QTpRcPEz6Qe+xmZhXjxG5mVjEDW4rpxD0TW9mnP0JaqxoNpasfNdPKfVSLUEZpZtC5x25mVjFO7GZmFePEbmZWMQNbY89T1+tEHd6sLJ2op/vfSG9yj93MrGKc2M3MKmagSjEnTH6Ke/3R0QZEJyYN68Yw4Txl0ryzJg4q99jNzCrGid3MrGKc2M3MKmagauy9YFT98NoDF/PURV1ftDw6Mbyx3ZvR5Nmntc89djOzinFiNzOrGJdiepzLLtaqIt47vXpfVRtfKT12SVMkrZe0Lfk9eYx2S5M22yQtzdg+JOnhzkdsZtY/yirFrAA2RMRcYEOyfABJU4DLgZOB+cDl6f8AJL0XeK474ZqZ9Y+yEvsiYHXyeDVwdkabBcD6iNgdEXuA9cBCAEmvAj4GXNWFWM3M+kpZNfajI2IXQETsknRURpvpwI7U8kiyDuBK4L8Bzzc6kKRlwDKAWdO7/3JH3SHp2ux2Y8kz3LHZOmgn7qTTie8CenWoZyfqzr3wunqZz09zOpbpJN0JvDZj02V5d5GxLiSdBMyJiL+WNLvRTiJiJbASYN6JkyLnsc3M+lbHEntEnDbWNkm/kDQt6a1PA57MaDYCnJJangFsBP4I+NeSfkot/qMkbYyIUzAzs9JKMUPAUuDq5PcdGW3WAZ9JfWF6BnBpROwG/idA0mP/lpN6czpRSsjaZ/1Vib164+5eHNJXVhmq2XORddWob15dvrK+PL0aOF3SNuD0ZBlJ8yTdAJAk8CuBTcnPFck6MzMbRyk99oh4Bjg1Y/1m4MOp5VXAqnH281PgzR0I0cysb/nK0y5r9uN0P48GqP/I/Qa6fyVkJ2420Qm9Elej8+V7nPYHzxVjZtYHJC2UtFXSsKRRF3WmObGbmfU4SROA64EzgeOB8yUdP1Z7J3Yzs943HxiOiO0R8SJwM7Ur+DO5xl6gVobz1dcw+6UmnMeo4Y6XjH9+ujUM0/Krfz/W/w3X7Xyg7X1aLllX4p88VmMndjOz8k2VtDm1vDK5an6/zCvxx9qZE7uZWfmejoh542wfAWamlmcAO8dq7MRuhcgaBteJ4Y5VlWsY4bkHLvbClby9EMOA2ATMlXQs8HNgCfBnYzV2Yjcz63ERsU/SRdSmWpkArIqILWO1d2I3M+sDEbEWWJunrYc7mplVjHvsBcoa+tVsDbKs4Xn1Nd4iZuBrNNxxkIciFnFpfqOhhq5/Dy732M3MKsaJ3cysYlyKKVA/f/TtxM0PRu2zyfu99gvPeGi9xj12M7OKcWI3M6sYl2IKVMSoGOs/rZSxOlG+8XvN9nOP3cysYpzYzcwqxondzKxiXGO3jhl15WmfXIVbhDKGQLZy0wvX5auplB67pCmS1kvalvyePEa7pUmbbZKWptYfLGmlpMck/UTS+7oXvZlZbyurFLMC2BARc4ENyfIBJE0BLqd2+6f5wOWp/wAuA56MiOOo3dj1e12J2sysD5RVilkEnJI8Xg1sBD5e12YBsD4idgNIWg8sBG4CPgS8ESAifgs8neegjz14SKEfPes/+ubZd7+UF5q9L2Wj+5lCK6WCxu07ccVss3rlylOXVWy/snrsR0fELoDk91EZbbJu3jpd0uHJ8pWS7pd0m6SjxzqQpGWSNkva/Bv2FhW/mVnP6lhil3SnpIczfhbl3UXGuqD2KWMG8P2I+APgB8DnxtpJRKyMiHkRMe+VTGz6dZiZ9ZuOlWIi4rSxtkn6haRpEbFL0jTgyYxmI/yuXAO1ZL4ReAZ4Hrg9WX8bcEERMZuZVUFZNfYhYClwdfL7jow264DPpL4wPQO4NCJC0jepJf27gFOBRzoecYYq1zTrvwtotuZehDznd3hnc/XtrsxiSft19zxxlvE3sf5QVo39auB0SduA05NlJM2TdANA8qXpldTuzr0JuGL/F6nUvmj9lKQHgfcDf9Pl+M3MelYpPfaIeIZaT7t+/Wbgw6nlVcCqjHY/A/6kkzGamfUrX3nahvrhe70w9M7Gl1Ui6VZ5xqxbPFeMmVnFOLGbmVWMSzEFyjMSwh/Ry5U50qai92K1weUeu5lZxTixm5lVjBO7mVnFuMbehirXy3vhqsZWbhzRyj7n3Fr+jJu9cL6tOtxjNzOrGCd2M7OKUUSUHUPXSHoK+FmXDzuVnDcC6bJejMsx5eOY8msnrtdFxJGtHljSd5Pj5/F0RCxs9Vijjj1Iib0MkjZHxLyy46jXi3E5pnwcU369GlenuRRjZlYxTuxmZhXjxN55K8sOYAy9GJdjyscx5dercXWUa+xmZhXjHruZWcU4sRdA0jmStkj6raQxv4GXtFDSVknDklak1h8r6R5J2yTdIungAmKaIml9ss/1qXvHptu8U9KPUz8vSDo72fZVSU+ktp3Ubkx540ravZQ69lBqfVnn6iRJP0j+zg9KOi+1rbBzNdZ7JLV9YvK6h5PzMDu17dJk/VZJC1qNoYWYPibpkeS8bJD0utS2zL9jF2L6oKSnUsf+cGrb0uRvvU3S0qJi6ikR4Z82f4B/Bfw+sBGYN0abCcDjwOuBg4EHgOOTbbcCS5LHXwQ+WkBMnwVWJI9XANc0aD8F2A0ckix/FVjcgXOVKy7guTHWl3KugOOAucnjY4BdwOFFnqvx3iOpNv8O+GLyeAlwS/L4+KT9RODYZD8TuhTTO1Pvm4/uj2m8v2MXYvogcN0Y7/Ptye/JyePJRcdY9o977AWIiEcjYmuDZvOB4YjYHhEvAjcDiyQJeBewJmm3Gji7gLAWJfvKu8/FwHci4vkCjj2eZuN6WZnnKiIei4htyeOdwJNAyxevjCHzPTJOrGuAU5Pzsgi4OSL2RsQTwHCyv47HFBF3p943PwRmFHDctmIaxwJgfUTsjog9wHqgsAuDeoUTe/dMB3aklkeSdUcA/y8i9tWtb9fREbELIPl9VIP2S4Cb6tb9bfLx+guSJhYQUzNxTZK0WdIP95eH6JFzJWk+tZ7i46nVRZyrsd4jmW2S8/AstfOS57mdiintAuA7qeWsv2O3Ynpf8jdZI2lmk8/ta57dMSdJdwKvzdh0WUTckWcXGetinPVtxZTn+an9TANOANalVl8K/DO1BLYS+DhwRRfjmhUROyW9HrhL0kPALzPalXGuvgYsjYjfJqtbPlf1u89YV//6Cn8fNZB7v5L+ApgHvCO1etTfMSIez3p+wTF9E7gpIvZKupDap5x35Xxu33NizykiTmtzFyPAzNTyDGAntXksDpd0UNID27++rZgk/ULStIjYlSSjJ8fZ1bnA7RHxm9S+dyUP90r6CrA8T0xFxZWUO4iI7ZI2Am8FvkGJ50rSa4BvA5+IiJfn2W3nXNUZ6z2S1WZE0kHAYdS+G8nz3E7FhKTTqP0n+Y6I2Lt//Rh/x3YTe8OYIuKZ1OLfA9eknntK3XM3thlPz3Eppns2AXOTUR0HUyt9DEXtG527qdW4AZYCeT4BNDKU7CvPPs+nrgyTJLj9de2zgYcLiClXXJIm7y9nSJoKvB14pMxzlfzNbgf+ISJuq9tW1LnKfI+ME+ti4K7kvAwBS5JRM8cCc4F7W4yjqZgkvRX4EnBWRDyZWp/5d+xSTNNSi2cBjyaP1wFnJLFNBs7gwE+q1VD2t7dV+AH+lFpPYC/wC2Bdsv4YYG2q3buBx6j1WC5LrX89tX+Ew8BtwMQCYjoC2ABsS35PSdbPA25ItZsN/Bx4Rd3z7wIeopakbgReVdC5ahgX8G+SYz+Q/L6g7HMF/AXwG+DHqZ+Tij5XWe8RamWds5LHk5LXPZych9ennntZ8rytwJkFvr8bxXRn8r7ff16GGv0duxDTfwW2JMe+G3hj6rkfSs7fMPCXRcXUSz++8tTMrGJcijEzqxgndjOzinFiNzOrGCd2M7OKcWI3M6sYJ3YbWJIulPSB5PEHJR2T2naDpOPLi86sdR7uaAYkV0Uuj4jNZcdi1i732K0vSZot6SeSVqcmejpE0qmSfiTpIUmrUlc+Xp2aM/xzybpPSVouaTG1i5G+nszd/XuSNiqZW1/S+cn+HpZ0TSqG5yT9raQHkkmuji7jXJjVc2K3fvb7wMqIeAu1CcI+Rm1u9PMi4gRqcyF9VNIUalcHvylpe1V6JxGxBtgM/HlEnBQR/7J/W1KeuYbaBFInAW9LzVJ4KPDDiDgR+CfgIx17pWZNcGK3frYjIr6fPL4ROBV4IiIeS9atBv6EWtJ/AbhB0nuBZuacfxuwMSKeitrEY19P9gnwIvCt5PF91KZnMCudE7v1s1xfECUJeT612SHPBr7bxDGypnnd7zfxuy+pXsKzpVqPcGK3fjZL0h8lj8+nNhnVbElzknXvB74n6VXAYRGxFriEWkml3q+AV2esvwd4h6SpkiYkx/lekS/CrGjuYVg/exRYKulL1GZmvJjardluS+Yq30TtvqhTgDskTaLWA//rjH19FfiipH8B9v9nQdTmaL+U2gyBojZbZxFTBZt1jIc7Wl+SNBv4VkS8ueRQzHqOSzFmZhXjHruZWcW4x25mVjFO7GZmFePEbmZWMU7sZmYV48RuZlYxTuxmZhXz/wHnxjGl5jdzDAAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "poses = np.linspace(env.unwrapped.min_position, env.unwrapped.max_position, 256)\n",
    "vels = np.linspace(-env.unwrapped.max_speed, env.unwrapped.max_speed, 256)\n",
    "positions, velocities = np.meshgrid(poses, vels)\n",
    "\n",
    "@np.vectorize\n",
    "def decide(position, velocity):\n",
    "    return agent.decide((position, velocity))\n",
    "\n",
    "action_values = decide(positions, velocities)\n",
    "\n",
    "fig, ax = plt.subplots()\n",
    "c = ax.pcolormesh(positions, velocities, action_values)\n",
    "ax.set_xlabel('position')\n",
    "ax.set_ylabel('velocity')\n",
    "fig.colorbar(c, ax=ax, boundaries=[-.5, .5, 1.5, 2.5], ticks=[0, 1, 2])\n",
    "fig.show();"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Save Agent"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pickle\n",
    "with open('./resources/agent.pkl', 'wb') as file:\n",
    "    pickle.dump(agent, file)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "env.close()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
